# import libraries
import pandas as pd
import plotly.express as px
from datetime import datetime, timedelta
import seaborn as sns
from matplotlib import pyplot as plt
import numpy as np
from scipy import stats as st
%matplotlib inline
import plotly.graph_objects as go
import math as mth
from statsmodels.stats.proportion import proportions_ztest
import warnings
warnings.filterwarnings('ignore')
#function for primary analysis of the dataset
def dataset (dataset):
dataset.columns = [x.lower().replace(' ', '_') for x in dataset.columns.values]
display(dataset.info())
print('*'*50)
display(dataset.describe())
print('*'*50)
display(dataset.sample(10))
print('*'*50)
display('Number of dublicate values', dataset.duplicated().sum())
#uploading data
events = pd.read_csv('/datasets/final_ab_events.csv', parse_dates=['event_dt'])
marketing_events = pd.read_csv('/datasets/ab_project_marketing_events.csv', parse_dates=['start_dt', 'finish_dt'])
new_users = pd.read_csv('/datasets/final_ab_new_users.csv', parse_dates=['first_date'])
participants = pd.read_csv('/datasets/final_ab_participants.csv')
dataset(events)
<class 'pandas.core.frame.DataFrame'> RangeIndex: 440317 entries, 0 to 440316 Data columns (total 4 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 user_id 440317 non-null object 1 event_dt 440317 non-null datetime64[ns] 2 event_name 440317 non-null object 3 details 62740 non-null float64 dtypes: datetime64[ns](1), float64(1), object(2) memory usage: 13.4+ MB
None
**************************************************
| details | |
|---|---|
| count | 62740.000000 |
| mean | 23.877631 |
| std | 72.180465 |
| min | 4.990000 |
| 25% | 4.990000 |
| 50% | 4.990000 |
| 75% | 9.990000 |
| max | 499.990000 |
**************************************************
| user_id | event_dt | event_name | details | |
|---|---|---|---|---|
| 338089 | C88FA2A19F927C1E | 2020-12-18 06:31:38 | login | NaN |
| 243909 | 8C819478F2623906 | 2020-12-27 03:23:51 | product_page | NaN |
| 382124 | 1D6A02B0987CD71A | 2020-12-22 01:09:55 | login | NaN |
| 70999 | 09FE8ABDED4AF737 | 2020-12-11 06:33:26 | product_cart | NaN |
| 405933 | B578A3850EBC557E | 2020-12-24 16:46:44 | login | NaN |
| 173103 | C5B25E28ED032491 | 2020-12-16 23:18:35 | product_page | NaN |
| 260336 | AB139261F9FEF6B5 | 2020-12-08 01:42:10 | login | NaN |
| 182583 | 87D6E4C509D5EBF1 | 2020-12-18 00:32:28 | product_page | NaN |
| 116462 | 7FDCDE6A910C6EA2 | 2020-12-25 08:38:33 | product_cart | NaN |
| 122570 | E69F98E44CAA2913 | 2020-12-28 07:24:49 | product_cart | NaN |
**************************************************
'Number of dublicate values'
0
There are gaps in the details column, which is normal, because this field contains additional information about the event.
dataset(marketing_events)
<class 'pandas.core.frame.DataFrame'> RangeIndex: 14 entries, 0 to 13 Data columns (total 4 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 name 14 non-null object 1 regions 14 non-null object 2 start_dt 14 non-null datetime64[ns] 3 finish_dt 14 non-null datetime64[ns] dtypes: datetime64[ns](2), object(2) memory usage: 576.0+ bytes
None
**************************************************
| name | regions | start_dt | finish_dt | |
|---|---|---|---|---|
| count | 14 | 14 | 14 | 14 |
| unique | 14 | 6 | 14 | 14 |
| top | St. Valentine's Day Giveaway | APAC | 2020-07-04 00:00:00 | 2020-03-10 00:00:00 |
| freq | 1 | 4 | 1 | 1 |
| first | NaN | NaN | 2020-01-25 00:00:00 | 2020-02-07 00:00:00 |
| last | NaN | NaN | 2020-12-30 00:00:00 | 2021-01-07 00:00:00 |
**************************************************
| name | regions | start_dt | finish_dt | |
|---|---|---|---|---|
| 10 | CIS New Year Gift Lottery | CIS | 2020-12-30 | 2021-01-07 |
| 2 | St. Patric's Day Promo | EU, N.America | 2020-03-17 | 2020-03-19 |
| 9 | Victory Day CIS (May 9th) Event | CIS | 2020-05-09 | 2020-05-11 |
| 13 | Chinese Moon Festival | APAC | 2020-10-01 | 2020-10-07 |
| 4 | 4th of July Promo | N.America | 2020-07-04 | 2020-07-11 |
| 5 | Black Friday Ads Campaign | EU, CIS, APAC, N.America | 2020-11-26 | 2020-12-01 |
| 3 | Easter Promo | EU, CIS, APAC, N.America | 2020-04-12 | 2020-04-19 |
| 11 | Dragon Boat Festival Giveaway | APAC | 2020-06-25 | 2020-07-01 |
| 6 | Chinese New Year Promo | APAC | 2020-01-25 | 2020-02-07 |
| 7 | Labor day (May 1st) Ads Campaign | EU, CIS, APAC | 2020-05-01 | 2020-05-03 |
**************************************************
'Number of dublicate values'
0
dataset(new_users)
<class 'pandas.core.frame.DataFrame'> RangeIndex: 61733 entries, 0 to 61732 Data columns (total 4 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 user_id 61733 non-null object 1 first_date 61733 non-null datetime64[ns] 2 region 61733 non-null object 3 device 61733 non-null object dtypes: datetime64[ns](1), object(3) memory usage: 1.9+ MB
None
**************************************************
| user_id | first_date | region | device | |
|---|---|---|---|---|
| count | 61733 | 61733 | 61733 | 61733 |
| unique | 61733 | 17 | 4 | 4 |
| top | EDDF307171589F79 | 2020-12-21 00:00:00 | EU | Android |
| freq | 1 | 6290 | 46270 | 27520 |
| first | NaN | 2020-12-07 00:00:00 | NaN | NaN |
| last | NaN | 2020-12-23 00:00:00 | NaN | NaN |
**************************************************
| user_id | first_date | region | device | |
|---|---|---|---|---|
| 51998 | 58B7D471C59B2EF4 | 2020-12-19 | APAC | Android |
| 9813 | 37DD338227177554 | 2020-12-14 | EU | PC |
| 18527 | 5732FDB1C948FE26 | 2020-12-08 | EU | Android |
| 43292 | 2BA3E36AAFBDFD01 | 2020-12-18 | EU | PC |
| 8241 | 3A0B362A5E4BABC3 | 2020-12-14 | EU | PC |
| 16828 | CDBE367CF4492D7E | 2020-12-21 | EU | Android |
| 21446 | 4F79F4AFE44091C9 | 2020-12-15 | CIS | Android |
| 49791 | 775AF16EA3BDE170 | 2020-12-19 | EU | Android |
| 21794 | DC1C43084F6156E5 | 2020-12-15 | CIS | PC |
| 54139 | FB4C0EEC427DCB62 | 2020-12-13 | APAC | Android |
**************************************************
'Number of dublicate values'
0
dataset(participants)
<class 'pandas.core.frame.DataFrame'> RangeIndex: 18268 entries, 0 to 18267 Data columns (total 3 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 user_id 18268 non-null object 1 group 18268 non-null object 2 ab_test 18268 non-null object dtypes: object(3) memory usage: 428.3+ KB
None
**************************************************
| user_id | group | ab_test | |
|---|---|---|---|
| count | 18268 | 18268 | 18268 |
| unique | 16666 | 2 | 2 |
| top | 823E881A8B197337 | A | interface_eu_test |
| freq | 2 | 9655 | 11567 |
**************************************************
| user_id | group | ab_test | |
|---|---|---|---|
| 1213 | A662A4DFC7F13881 | A | recommender_system_test |
| 13201 | 69C16A396D80FB81 | B | interface_eu_test |
| 2083 | A35EA209D6186DB7 | B | recommender_system_test |
| 10808 | 5C530719759F0456 | A | interface_eu_test |
| 6775 | 0432FE839A9A354C | B | interface_eu_test |
| 17923 | 9BA3280B1576B4E9 | B | interface_eu_test |
| 4594 | B54431CE7FDB94E0 | B | recommender_system_test |
| 9142 | 1AE6FF352875CA8C | B | interface_eu_test |
| 13988 | 20E8EBC0133C64EE | A | interface_eu_test |
| 3991 | 107D9F8613EBF2AA | B | recommender_system_test |
**************************************************
'Number of dublicate values'
0
#let's check for which period the event data is provided
print('Date of the first event', events['event_dt'].min())
print('Data of the last event', events['event_dt'].max())
Date of the first event 2020-12-07 00:00:33 Data of the last event 2020-12-30 23:36:33
The technical assignment says that the data contains all the events of new users in the period from December 7, 2020 to January 4, 2021, however, we see that the data is limited to December 30, the test was stopped ahead of time.
#let's check for what period the data is provided in the file with information about new users
print('The first date of registration', new_users['first_date'].min())
print('The last date of registration', new_users['first_date'].max())
The first date of registration 2020-12-07 00:00:00 The last date of registration 2020-12-23 00:00:00
The technical assignment says that the file contains users who registered in the online store in the period from December 7 to December 21, 2020, but we see that there are also users with a registration date after December 21. The test was stopped on December 30, respectively, not all users managed to "live" all 14 days from the registration date. We will remove users who came after December 16:
#checking how many test there are in data
participants['ab_test'].unique()
array(['recommender_system_test', 'interface_eu_test'], dtype=object)
#let's split the dataframe with participants into 2 sets
recommender_system_test = participants.query('ab_test=="recommender_system_test"')
interface_eu_test = participants.query('ab_test=="interface_eu_test"')
#adding information about regions to the dataframe of test participants
recommender_system_test = pd.merge(new_users, recommender_system_test, how="inner", on='user_id')
recommender_system_test['first_date'].max()
Timestamp('2020-12-21 00:00:00')
#calculation of the share of users from the EU
share = len(recommender_system_test.query('region=="EU"')['user_id'])/len(new_users.query('region=="EU"')['user_id'])
print("{0:.2%}".format(share))
13.73%
According to the terms of the TOR, 15% of new users from the EU should participate in the test, it was possible to score 13.73%. Let's check how statistically significant this difference is for our test.
H0 - the shares of users in the sample and the general population from the EU are equal. H1 - there is a difference between the shares.
count = round(len(new_users.query('region=="EU"')['user_id'])*0.15)
nobs = len(new_users.query('region=="EU"')['user_id'])
stat, pval = proportions_ztest(count, nobs, share)
print('{0:0.3f}'.format(pval))
0.000
There is no reason to consider the shares are different.
recommender_system_test.groupby(by=['group'])['user_id'].nunique()
group A 3824 B 2877 Name: user_id, dtype: int64
print('Total number of test participants', recommender_system_test['user_id'].nunique())
Total number of test participants 6701
The number of test participants is more than expected (6000), the groups are not divided 50/50 by the number of participants. Since we are studying conversion, a relative metric, the groups may be unbalanced.
begin_date = pd.to_datetime("2020-12-07 00:00:00")
end_date = pd.to_datetime("2020-12-30 23:59:59")
temp = marketing_events[(marketing_events['start_dt'] >= begin_date) | (marketing_events['finish_dt'] >= end_date)]
temp
| name | regions | start_dt | finish_dt | |
|---|---|---|---|---|
| 0 | Christmas&New Year Promo | EU, N.America | 2020-12-25 | 2021-01-03 |
| 10 | CIS New Year Gift Lottery | CIS | 2020-12-30 | 2021-01-07 |
There is an intersection of the time of the test with two marketing activities related to the New Year holidays. Nevertheless, since the start date of the campaign is close to the end of the test and at the end of December, user behavior is in principle difficult to call standard due to the increase in activity before the holidays, we will leave the events from these ranges. The test was aimed at users from the EU, so the CIS New Year Gift Lottery promotion should not affect it.
Let's make sure that there are no intersections with a competing test and there are no users participating in two test groups at the same time. We will also check the uniformity of the distribution of users into test groups and the correctness of their formation.
#let's divide the participants into 2 lists according to the tests
participants_recommender = recommender_system_test['user_id']
participants_interface = interface_eu_test['user_id']
#function for determining the intersection between lists
def intersection(l1, l2):
return list(set(l1) & set(l2))
print('The number of participants who got into both tests', len(intersection(participants_recommender, participants_interface)))
print('Share of the total number of participants', "{0:.2%}".format(
len(intersection(participants_recommender, participants_interface))/len(recommender_system_test['user_id']))
)
The number of participants who got into both tests 1602 Share of the total number of participants 23.91%
The two tests are competing and both affect the funnel. If we delete the users who got into both tests, we will reduce the number of users by 24.36%, which is too much data loss.
recommender_system_test_a = recommender_system_test.query('group == "A"')
recommender_system_test_b = recommender_system_test.query('group == "B"')
#let's divide the participants into 2 lists by test groups
participants_recommender_a = recommender_system_test_a['user_id']
participants_recommender_b = recommender_system_test_b['user_id']
print('The number of participants in both groups', len(intersection(participants_recommender_a, participants_recommender_b)))
The number of participants in both groups 0
fig = px.histogram(recommender_system_test, x="first_date",
color='group', barmode='group',
height=400, width=800)
fig.update_layout(barmode='stack',
xaxis={'categoryorder':'total descending'},
title_text='Distribution of users by registration date between groups',
yaxis=dict(
title='Number of users',
titlefont_size=16,
tickfont_size=14,
))
fig.show()
The overall pattern is the same.
fig = px.histogram(recommender_system_test, x="region",
color='group', barmode='group',
height=400, width=800)
fig.update_layout(barmode='stack',
xaxis={'categoryorder':'total descending'},
title_text='Distribution of users in the test by region between groups',
yaxis=dict(
title='Number of users',
titlefont_size=16,
tickfont_size=14,
))
fig.show()
The test was aimed at an audience from the EU, so it is expected that the majority of users are from there. Other regions, apparently, were hit due to an error in the mechanism for recruiting users into groups. Let's filter out the test participants so that only users from other regions remain.
recommender_system_test = recommender_system_test.query('region=="EU"')
fig = px.histogram(recommender_system_test, x="device",
color='group', barmode='group',
height=400, width=800)
fig.update_layout(barmode='stack',
xaxis={'categoryorder':'total descending'},
title_text='Distribution of users by device between groups',
yaxis=dict(
title='Number of users',
titlefont_size=16,
tickfont_size=14,
))
fig.show()
The distribution is uniform across the devices.
#let's add events from the events dataset to the dataset with test participants
events_test = pd.merge(recommender_system_test, events, how="inner", on='user_id')
events_test.head(5)
| user_id | first_date | region | device | group | ab_test | event_dt | event_name | details | |
|---|---|---|---|---|---|---|---|---|---|
| 0 | D72A72121175D8BE | 2020-12-07 | EU | PC | A | recommender_system_test | 2020-12-07 21:52:10 | product_page | NaN |
| 1 | D72A72121175D8BE | 2020-12-07 | EU | PC | A | recommender_system_test | 2020-12-07 21:52:07 | login | NaN |
| 2 | DD4352CDCF8C3D57 | 2020-12-07 | EU | Android | B | recommender_system_test | 2020-12-07 15:32:54 | product_page | NaN |
| 3 | DD4352CDCF8C3D57 | 2020-12-07 | EU | Android | B | recommender_system_test | 2020-12-08 08:29:31 | product_page | NaN |
| 4 | DD4352CDCF8C3D57 | 2020-12-07 | EU | Android | B | recommender_system_test | 2020-12-10 18:18:27 | product_page | NaN |
events_test['event_day_week'] = pd.DatetimeIndex(events_test['event_dt']).day_name()
week = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
events_test['event_day_week'] = pd.Categorical(events_test['event_day_week'], categories=week, ordered=True)
events_test = events_test.sort_values('event_day_week')
fig = px.histogram(events_test, x="event_day_week",
color='group', barmode='group',
height=400, width=800)
fig.update_layout(barmode='stack',
title_text='Distribution of events between groups by days of the week',
yaxis=dict(
title='Number of users',
titlefont_size=16,
tickfont_size=14,
))
fig.show()
Most of the events occur on Monday.
#let's count the number of events by users
eventsByUsersA = events_test.query('group == "A"').groupby('user_id', as_index=False).agg({'event_dt': 'count'})
eventsByUsersB = events_test.query('group == "B"').groupby('user_id', as_index=False).agg({'event_dt': 'count'})
eventsByUsersA.columns = ['user_id', 'events']
eventsByUsersB.columns = ['user_id', 'events']
eventsByUsersA.head(5)
| user_id | events | |
|---|---|---|
| 0 | 0010A1C096941592 | 12 |
| 1 | 00341D8401F0F665 | 2 |
| 2 | 003DF44D7589BBD4 | 15 |
| 3 | 00505E15A9D81546 | 5 |
| 4 | 006E3E4E232CE760 | 6 |
fig = plt.figure(figsize=(15,7))
x = eventsByUsersA['events']
y = eventsByUsersB['events']
bins = 30
plt.title('Distribution of the number of events per user in groups')
plt.grid(color='b', linestyle='-', linewidth=0.1)
plt.hist(x, bins, alpha=0.9, label='Group А')
plt.hist(y, bins, alpha=0.5, label='Group В')
plt.legend(loc='upper right')
plt.show()
The distributions are similar. Let's use the t-test to check the equality of the average number of events per user in two groups.
sample_1 = eventsByUsersA['events']
sample_2 = eventsByUsersB['events']
alpha = .05
results = st.ttest_ind(
sample_1,
sample_2)
print('p-value:', results.pvalue)
if results.pvalue < alpha:
print("Reject the null hypothesis")
else:
print("Cannot reject the null hypothesis")
p-value: 4.896419370297101e-16 Reject the null hypothesis
print('The average number of events per user in Group A:', sample_1.mean())
print('The average number of events per user in Group B:', sample_2.mean())
diff = (sample_1.mean() - sample_2.mean())/sample_1.mean()
print('Difference:', "{0:.2%}".format(diff))
The average number of events per user in Group A: 7.03110599078341 The average number of events per user in Group B: 5.827822120866591 Difference: 17.11%
The test showed that there is a difference between the two aggregates. The average number of events per user in group B is 17% more than in group A. Perhaps the changes were positive and in group B users go further down the sales funnel.
fig = plt.figure(figsize=(15,7))
events_test.query('group == "A"')['event_dt'].hist(density=True, alpha=0.9, label="Группа А", bins=50)
events_test.query('group == "B"')['event_dt'].hist(density=True, bins=50, alpha=0.5, color='pink', label="Group В")
plt.title('Histogram of the number of observations by date')
plt.legend(loc="upper right")
plt.ylabel('Number of observations')
plt.show()
In Group B, users were more active at the beginning of the test. In both groups, activity dropped after the peak on December 21. In Group A, we see a sharp increase on the 14th. It is possible that a failure occurred on this day, which affects the results of the analysis.
events_test_a = events_test.query('group == "A"')
events_test_b = events_test.query('group == "B"')
funnel_a = events_test_a.groupby('event_name').agg({'user_id':'nunique'})\
.sort_values(by='user_id', ascending=False).reset_index(level=[0,0])
#percentage of the total number of unique users
funnel_a['percent'] = funnel_a['user_id'] / events_test_a ['user_id'].nunique()
funnel_a['conversion'] = funnel_a['user_id']/funnel_a['user_id'].shift(periods=1)
#the first event in the funnel is 100%
funnel_a = funnel_a.fillna(1.00)
print('Funnel of events in Group A')
with pd.option_context('display.float_format', '{:.2%}'.format):
display(funnel_a)
Funnel of events in Group A
| event_name | user_id | percent | conversion | |
|---|---|---|---|---|
| 0 | login | 2604 | 100.00% | 100.00% |
| 1 | product_page | 1685 | 64.71% | 64.71% |
| 2 | purchase | 833 | 31.99% | 49.44% |
| 3 | product_cart | 782 | 30.03% | 93.88% |
#let's change the order of events so that the target action - purchase - is at the end
funnel_a = funnel_a.set_index('event_name')
new_index = ['login', 'product_page', 'product_cart', 'purchase']
funnel_a = funnel_a.reindex(new_index)
funnel_a = funnel_a.reset_index()
print('Funnel of events in Group A')
with pd.option_context('display.float_format', '{:.2%}'.format):
display(funnel_a)
Funnel of events in Group A
| event_name | user_id | percent | conversion | |
|---|---|---|---|---|
| 0 | login | 2604 | 100.00% | 100.00% |
| 1 | product_page | 1685 | 64.71% | 64.71% |
| 2 | product_cart | 782 | 30.03% | 93.88% |
| 3 | purchase | 833 | 31.99% | 49.44% |
# funnel chart
fig = go.Figure(
go.Funnel(
y = funnel_a['event_name'],
x = funnel_a['user_id'],
)
)
fig.update_layout(title_text='Event funnel for group A')
fig.show()
funnel_b = events_test_b.groupby('event_name').agg({'user_id':'nunique'})\
.sort_values(by='user_id', ascending=False).reset_index(level=[0,0])
#percentage of the total number of unique users
funnel_b['percent'] = funnel_b['user_id'] / events_test_b ['user_id'].nunique()
funnel_b['conversion'] = funnel_b['user_id']/funnel_b['user_id'].shift(periods=1)
#the first event in the funnel is 100%
funnel_b = funnel_b.fillna(1.00)
#let's change the order of events so that the target action - purchase - is at the end
funnel_b = funnel_b.set_index('event_name')
new_index = ['login', 'product_page', 'product_cart', 'purchase']
funnel_b = funnel_b.reindex(new_index)
funnel_b = funnel_b.reset_index()
print('Event funnel for group B')
with pd.option_context('display.float_format', '{:.2%}'.format):
display(funnel_b)
Event funnel for group B
| event_name | user_id | percent | conversion | |
|---|---|---|---|---|
| 0 | login | 877 | 100.00% | 100.00% |
| 1 | product_page | 493 | 56.21% | 56.21% |
| 2 | product_cart | 244 | 27.82% | 97.99% |
| 3 | purchase | 249 | 28.39% | 50.51% |
# funnel chart
fig = go.Figure(
go.Funnel(
y = funnel_b['event_name'],
x = funnel_b['user_id'],
)
)
fig.update_layout(title_text='Event funnel for group B')
fig.show()
Group B shows a worse conversion rate at the second stage of the funnel. The conversion rate at the last stage of the funnel is almost the same for both groups. There are more "purchase" events for both group A and group B than basket views - perhaps some buyers make a "quick purchase in 1 click", bypassing the basket view.
H0 - there are no differences between the shares, H1 - there are differences between the shares.
To test hypotheses, we will use the z-criterion.
#function for conducting the test
def test(successes1, successes2, trials1, trials2, alpha):
p1 = successes1 / trials1
p2 = successes2 / trials2
# the proportion of success in the combined dataset:
p_combined = (successes1 + successes2) / (trials2 + trials1)
# the difference in proportions in datasets
difference = p1 - p2
# we count statistics in the standard deviations of the standard normal distribution
z_value = difference / mth.sqrt(p_combined * (1 - p_combined) * (1/trials1 + 1/trials2))
# we set the standard normal distribution (mean 0, standard deviation 1)
distr = st.norm(0, 1)
p_value = (1 - distr.cdf(abs(z_value))) * 2
print('p-value: ', p_value)
if p_value < alpha:
print('We reject the null hypothesis: there is a significant difference between the shares')
else:
print('It was not possible to reject the null hypothesis, there is no reason to consider the shares are different')
users_total_a = events_test_a['user_id'].nunique()
users_total_b = events_test_b['user_id'].nunique()
events = ['product_page', 'product_cart', 'purchase']
funnel_a = funnel_a.set_index('event_name')
funnel_b = funnel_b.set_index('event_name')
for event in events:
print('Test for the event:', event)
test(funnel_a.loc[event, 'user_id'], funnel_b.loc[event, 'user_id'], users_total_a, users_total_b, 0.01)
print(' '*50)
Test for the event: product_page
p-value: 6.942739359416805e-06
We reject the null hypothesis: there is a significant difference between the shares
Test for the event: product_cart
p-value: 0.21469192029582396
It was not possible to reject the null hypothesis, there is no reason to consider the shares are different
Test for the event: purchase
p-value: 0.04652482738393027
It was not possible to reject the null hypothesis, there is no reason to consider the shares are different
According to the design of the test, there are a number of errors related to the timing of the test and the set of users - it is recommended to correct them during further tests.
After the test, group B did not show significant results, at the login->product_page stage, the conversion generally worsened.